{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "prod_version_random_forest_classification.ipynb",
      "provenance": [],
      "toc_visible": true,
      "mount_file_id": "1idRMHujiyn3-Kb_g-3ow3NteHFu7L9J5",
      "authorship_tag": "ABX9TyMF1CQKulEWJLaPKysSvfOX",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/monsund/unsupervised-classification-of-Linkedin-Profiles-using-KMeans-LDA-TFIDF/blob/master/prod_version_random_forest_classification.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iPVeUdv-flGv"
      },
      "source": [
        "import pandas as pd\n",
        "import pickle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YNsYWWGhftCp"
      },
      "source": [
        "ranforest_model_filepath = '/content/drive/My Drive/NLP/with_swaraj/model/ranforest_model.pkl'\n",
        "tfidf_model_filepath = '/content/drive/My Drive/NLP/with_swaraj/model/tfidf_model_profile_classification.pkl'"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zn8cKpbNYRwE"
      },
      "source": [
        "# **Import Model**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5wexiL2ja_yE"
      },
      "source": [
        "with open(ranforest_model_filepath, \"rb\") as fp:   # Unpickling\n",
        "  ranforest_model = pickle.load(fp)\n",
        "\n",
        "with open(tfidf_model_filepath, \"rb\") as fp:   # Unpickling\n",
        "  tfidf_model = pickle.load(fp)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gCLbI3DEbM3_",
        "outputId": "01d112b2-20be-4706-a4d2-308230153892",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 153
        }
      },
      "source": [
        "ranforest_model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RandomForestClassifier(bootstrap=True, ccp_alpha=0.0, class_weight=None,\n",
              "                       criterion='gini', max_depth=None, max_features='auto',\n",
              "                       max_leaf_nodes=None, max_samples=None,\n",
              "                       min_impurity_decrease=0.0, min_impurity_split=None,\n",
              "                       min_samples_leaf=1, min_samples_split=2,\n",
              "                       min_weight_fraction_leaf=0.0, n_estimators=100,\n",
              "                       n_jobs=None, oob_score=False, random_state=None,\n",
              "                       verbose=0, warm_start=False)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dlTvCRXNbWoa",
        "outputId": "6a140ca0-1b31-493e-bb47-80ce2d54dcb2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "tfidf_model"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TfidfVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
              "                dtype=<class 'numpy.float64'>, encoding='utf-8',\n",
              "                input='content', lowercase=True, max_df=1.0, max_features=None,\n",
              "                min_df=1, ngram_range=(1, 1), norm='l2', preprocessor=None,\n",
              "                smooth_idf=True, stop_words=None, strip_accents=None,\n",
              "                sublinear_tf=False, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
              "                tokenizer=None, use_idf=True, vocabulary=None)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tYvB8dRub7rt"
      },
      "source": [
        "# **Remove Punctuation**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FNcNlMnSb7sG"
      },
      "source": [
        "#Function for removing punctuation and lowering case\n",
        "def remove_punct(text):\n",
        "  # define punctuation\n",
        "  punctuations = '\\'!()-[]{};:\\\"\\,<>./?#$%^&*_~'\n",
        "  \n",
        "  for char in text:\n",
        "    if char in punctuations:\n",
        "      text = text.lower().replace(char,' ')    #lowering case\n",
        "      text = text.replace('  ', ' ')\n",
        "  return text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c2LS7XGTb7sO"
      },
      "source": [
        "# **Remove Stopwords**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RxJkeF55b7sP",
        "outputId": "8bd9fb7e-92d0-475c-87c3-e35614106699",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 85
        }
      },
      "source": [
        "import nltk\n",
        "nltk.download('punkt')\n",
        "from nltk.corpus import stopwords\n",
        "nltk.download('stopwords')\n",
        "from nltk.tokenize import word_tokenize"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Unzipping tokenizers/punkt.zip.\n",
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Unzipping corpora/stopwords.zip.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n6dmAUMJb7sS"
      },
      "source": [
        "# stop_words = set(stopwords.words('english'))\n",
        "# additional_stop_words = set(['technology', 'development', 'experience', 'project', 'company'])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "t2-SHsLNb7sV"
      },
      "source": [
        "# stop_words = stop_words.union(additional_stop_words)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r45vFG5Nb7sZ"
      },
      "source": [
        "# Function removing stopwords-----------\n",
        "def remove_stopwords(text):\n",
        "  stop_words = set(stopwords.words('english'))\n",
        "  additional_stop_words = set(['technology', 'development', 'experience', 'project', 'company'])\n",
        "  stop_words = stop_words.union(additional_stop_words)\n",
        "\n",
        "  word_tokens = word_tokenize(text)\n",
        "  filtered_text = [w for w in word_tokens if not w in stop_words]\n",
        "  data = (' '.join(filtered_text))\n",
        "  return data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Pjmq9f1LiYzR"
      },
      "source": [
        "# **TDIDF Vectorization**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YtzUJts2iOho"
      },
      "source": [
        "from sklearn.feature_extraction.text import TfidfVectorizer\n"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DOxNXRscRu7u"
      },
      "source": [
        "# **Function for finding Probability of Each Class**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "F5Nm50VcMuRV"
      },
      "source": [
        "def get_class_prob(y_prob_list):\n",
        "  topic_prob = []\n",
        "  for i in range(len(y_prob_list)):\n",
        "    topic3_prob = y_prob_list[i][0]\n",
        "    topic4_prob = y_prob_list[i][1]\n",
        "    topic5_prob = y_prob_list[i][2]\n",
        "    topic7_prob = y_prob_list[i][3]\n",
        "    topic8_prob = y_prob_list[i][4]\n",
        "    topic10_prob = y_prob_list[i][5]\n",
        "    topic11_prob = y_prob_list[i][6]\n",
        "    topic12_prob = y_prob_list[i][7]\n",
        "    topic14_prob = y_prob_list[i][8]\n",
        "    \n",
        "    dic = {'managerial field':topic3_prob, 'hr or people management':topic4_prob, 'marketing domain':topic5_prob, 'data science domain':topic7_prob,\n",
        "          'product visionnaire, product management related field':topic8_prob, 'software domain':topic10_prob, 'related to investment,startups looking for investment':topic11_prob,\n",
        "          'business,strategic growth,revenue':topic12_prob, 'entrepreneur or working with entrepreneur':topic14_prob}\n",
        "    sort_dic = sorted(dic.items(), key=lambda x: x[1], reverse=True)\n",
        "    topic_prob.append(sort_dic)\n",
        "  return topic_prob  "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "wNwnbl1RejTh"
      },
      "source": [
        "# **Model Prediction**"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zxm10259eBJz"
      },
      "source": [
        "def get_profile_info(model, text):\n",
        "  text_remove_punct = remove_punct(text)\n",
        "  text_remove_stopwords = remove_stopwords(text_remove_punct)\n",
        "  text_vec = tfidf_model.transform([text_remove_stopwords])\n",
        "  class_prob = model.predict_proba(text_vec)\n",
        "\n",
        "  profile_prob = get_class_prob(class_prob)\n",
        "  return profile_prob[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5G-xzANZaKbO"
      },
      "source": [
        "# topic_dic = {5: 'marketing domain', 8:'related to product visionnaire', 7:'related to data domain', 10:'software domain',14: 'entrepreneur or working with entrepreneur', \n",
        "#              11:'related to investment; startups looking for investment', 3:'managerial field', 12:'business,strategic growth,revenue', 4:'hr or people management'}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0DPetgAxeBDm",
        "outputId": "dc5706e5-3081-43d8-e68e-4bf41e0ffaf9",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 170
        }
      },
      "source": [
        "text = '''HR Professional; having over 6 years of experience in managing a full spectrum of human resources programs, services, and functions with proven expertise in implementing the policies and procedures. '''\n",
        "# tfidf_model.transform([text]).toarray()\n",
        "get_profile_info(ranforest_model,text)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('hr or people management', 0.43),\n",
              " ('software domain', 0.2),\n",
              " ('marketing domain', 0.18),\n",
              " ('managerial field', 0.09),\n",
              " ('data science domain', 0.05),\n",
              " ('product visionnaire, management related field', 0.03),\n",
              " ('related to investment,startups looking for investment', 0.01),\n",
              " ('business,strategic growth,revenue', 0.01),\n",
              " ('entrepreneur or working with entrepreneur', 0.0)]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 28
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LrswO2oetBBN"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}